{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-04T14:31:18.447215Z",
     "start_time": "2024-11-04T14:31:14.086633Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "sys.path.append(\"../\")"
   ]
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df =  pd.read_csv(\"../data/Himawari8_2023_2024_samplebased_valid.csv\")\n",
    "row_num = len(df)\n",
    "step = 10000\n",
    "for start in range(0, step, step):\n",
    "    stop = start + step\n",
    "    filename = \"../data/split/Himawari8_2023_2024_samplebased_valid.csv\"\n",
    "    d = df[start:stop]\n",
    "    d.to_csv(filename, index=None)\n",
    "    "
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-11-05T12:15:22.213215Z",
     "start_time": "2024-11-05T12:15:20.147372Z"
    }
   },
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true,
    "ExecuteTime": {
     "start_time": "2024-11-04T22:46:23.292802Z"
    }
   },
   "outputs": [],
   "source": [
    "from esidlm.learner.dsnet import DSNETLearner\n",
    "DSNET_TRAINING_CONFIG = {\n",
    "\n",
    "    \"global\": {\n",
    "        \"seed\": 42,\n",
    "        \"output_folder\": \"densenet\",\n",
    "    },\n",
    "\n",
    "    \"data\": {\n",
    "        \"train_data\": \"../data/Himawari8_2023_2024_samplebased_train.csv\",\n",
    "        \"valid_data\": \"../data/Himawari8_2023_2024_samplebased_valid.csv\",\n",
    "        \"test_data\": \"../data/Himawari8_2023_2024_samplebased_test.csv\",\n",
    "        \"cont_cols\":  ['band_1','band_2','band_3','band_4','band_5','band_6','band_7',\n",
    "                    'band_8','band_9','band_10','band_11','band_12','band_13','band_14',\n",
    "                    'band_15','band_16','band_17','band_18','band_19','band_20',\n",
    "                    'band_17_sin','band_18_sin','band_19_sin','band_20_sin',\n",
    "                    'band_17_cos','band_18_cos','band_19_cos','band_20_cos',\n",
    "                    'band_9_13', 'band_9_15', 'band_10_15', 'band_11_13', 'band_13_15'],\n",
    "        \"cate_cols\": ['hour_loc'],\n",
    "        \"target_col\":\"PM25\",\n",
    "    },\n",
    "\n",
    "    \"dataloader\": {\n",
    "        \"batch_size\": 512,\n",
    "        \"num_workers\": 4,\n",
    "    },\n",
    "\n",
    "    \"model\": {\n",
    "        \"net\": {\n",
    "            \"d_embed\": 4,\n",
    "            \"d_model\": 2,\n",
    "            \"n_layers\":2,\n",
    "            \"p_drop\": 0.1,\n",
    "            \"act_fn\": \"relu\"\n",
    "        },\n",
    "        \"optimizer\": {\n",
    "            \"lr\": 0.0003,\n",
    "            \"weight_decay\": 1e-4,\n",
    "        },\n",
    "    },\n",
    "\n",
    "    \"callback\": {\n",
    "        \"model_checkpoint\": {\n",
    "            \"save_top_k\": 1,\n",
    "            \"monitor\": \"valid_r2\",\n",
    "            \"mode\": \"max\",\n",
    "            \"verbose\": True\n",
    "        },\n",
    "        \"early_stopping\": {\n",
    "            \"monitor\": \"valid_r2\",\n",
    "            \"mode\": \"max\",\n",
    "            \"patience\": 800,\n",
    "            \"verbose\": True\n",
    "        }\n",
    "    },\n",
    "\n",
    "    \"trainer\": {\n",
    "        \"max_epochs\": 800,\n",
    "        \"accelerator\": \"cpu\",\n",
    "        \"devices\": 1,\n",
    "        \"deterministic\": True\n",
    "    }\n",
    "}\n",
    "learner = DSNETLearner(DSNET_TRAINING_CONFIG)\n",
    "learner.run_model_training()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from esidlm.learner.dsnet import DSNETLearner\n",
    "\n",
    "DSNET_INFERENCE_CONFIG = {\n",
    "    \"tif_config\":{\n",
    "        'cloud_mask': 1,\n",
    "        'csv2tif':1,\n",
    "    },\n",
    "    \"global\": {\n",
    "        \"seed\": 42,\n",
    "        \"output_folder\": \"/mnt/d/g/output2\",\n",
    "    },\n",
    "\n",
    "    \"data\": {\n",
    "        \"inference_folder\": \"../data/Himawari8_2023_2024_samplebased_test.csv\",\n",
    "        \"cont_cols\":  ['band_1','band_2','band_3','band_4','band_5','band_6','band_7',\n",
    "                    'band_8','band_9','band_10','band_11','band_12','band_13','band_14',\n",
    "                    'band_15','band_16','band_17','band_18','band_19','band_20',\n",
    "                    'band_17_sin','band_18_sin','band_19_sin','band_20_sin',\n",
    "                    'band_17_cos','band_18_cos','band_19_cos','band_20_cos',\n",
    "                    'band_9_13', 'band_9_15', 'band_10_15', 'band_11_13', 'band_13_15'],\n",
    "        \"cate_cols\": [\"hour_loc\"],\n",
    "        \"target_col\":\"PM25\",\n",
    "    },\n",
    "\n",
    "    \"dataloader\": {\n",
    "        \"batch_size\": 1024,\n",
    "        \"num_workers\": 4,\n",
    "    },\n",
    "\n",
    "    \"model\": {\n",
    "        \"model_checkpoint_path\": \"/mnt/d/g/output2/lightning_logs/version_11/checkpoints/epoch=79-step=18720.ckpt\"\n",
    "    },\n",
    "\n",
    "    \"trainer\": {\n",
    "        \"accelerator\": \"gpu\",\n",
    "        \"devices\": 1,\n",
    "    }\n",
    "}\n",
    "learner = DSNETLearner(DSNET_INFERENCE_CONFIG)\n",
    "learner.run_model_inference()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "alpha",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
